package com.blz.flare.parser.impl;

import au.id.jericho.lib.html.*;
import com.blz.flare.crawler.PostException;
import com.blz.flare.parser.ContentParser;
import com.blz.flare.parser.ParseResult;
import com.blz.flare.parser.ParseResultImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.*;
import java.util.regex.Pattern;

/**
 * HTTP implementation of the {@link ContentParser}.
 */
public class HTTPContentParser
        implements ContentParser {

    private static final Logger logger = LoggerFactory.getLogger(HTTPContentParser.class);

    private Map<String, String> metadataFields = new HashMap<String, String>();
    
    /**
     * {@inheritDoc}
     */
    @Override
    public boolean supports(String contentType) {
        return contentType.startsWith("http")
                ? true
                : false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public ParseResult parse(URI uri)
            throws  Exception {

        // parse some web content
        ParseResultImpl parseResult = new ParseResultImpl();
        Source source = null;
        try {
            source=new Source(uri.toURL());
        } catch (IOException ex) {
            logger.warn("error sourcing file: "+uri, ex);
            return parseResult;
        }

		// Call fullSequentialParse manually as most of the source will be parsed.
		source.fullSequentialParse();

        // grab the title
		String title=getTitle(source);

        // grab the 'out' links
        List<URI> newUrisToProcess = new ArrayList<URI>();
        parseResult.setParsedUris(getUrisFromContentSource(source, uri));


		String text = source.getTextExtractor().setIncludeAttributes(true).toString().trim();
		Map<String, String> metavalues = new HashMap<String, String>();
		String value;
		for (String meta : metadataFields.keySet()) {
			value = getMetaValue(source, meta);
			if (value != null) {
				value = value.trim();
				if (value.length() > 0) {
					metavalues.put(metadataFields.get(meta), value);
				}
			}
		}

        // get the data and the full text
        parseResult.setData(metavalues);
        TextExtractor extractor = new TextExtractor(source);
        parseResult.setText(extractor.toString());
        parseResult.setTitle(title);

        return parseResult;
    }

    /**
     * Get the title of a {@link Source}.
     *
     * @param source the source
     * @return the title
     */
	private static String getTitle(Source source) {
		Element titleElement=source.findNextElement(0,HTMLElementName.TITLE);
		if (titleElement==null) return null;
		// TITLE element never contains other tags so just decode it collapsing whitespace:
		return CharacterReference.decodeCollapseWhiteSpace(titleElement.getContent());
	}

    /**
     * Get the metavalue for a specific source key.
     *
     * @param source the source
     * @param key the key
     * @return the metavalue
     */
	private static String getMetaValue(Source source, String key) {
		for (int pos=0; pos<source.length();) {
			StartTag startTag=source.findNextStartTag(pos,"name",key,false);
			if (startTag==null) return null;
			if (startTag.getName()==HTMLElementName.META)
				return startTag.getAttributeValue("content"); // Attribute values are automatically decoded
			pos=startTag.getEnd();
		}
		return null;
	}

    /**
     * Get all the {@link URI}s in a {@link Source}.
     * @param source the source
     * @param uri the uri we're processing
     * @return a list of uris
     */
    private static List<URI> getUrisFromContentSource(Source source, URI uri) {

        List<URI> retUris = new ArrayList<URI>();

        List<Element> linkElements=source.findAllElements(HTMLElementName.A);
        URI linkuri;
		URL linkurl;

        // go over every possible element that could take us to another page
		for (Iterator<Element> i=linkElements.iterator(); i.hasNext();) {

            linkurl = null;
			Element linkElement=(Element)i.next();
			String href=linkElement.getAttributeValue("href");

            // skip null hrefs of javascript tags
            if (href==null || href.startsWith("javascript:")) {
                continue;
            }

            // get rid of empty anchors
			if (href.endsWith("#")) {
				href = href.substring(0, href.length() - 1);
			}

            // try to grab a proper url
			try {
				linkuri = new URI(href);
				linkurl = linkuri.toURL();
			} catch (Exception e) {
				// We may have a relative url
				try {
					linkurl = new URL(uri.toURL(), href);
				} catch (Exception e2) {
					logger.debug("Error parsing URL {} href", href, e2);
				}
			}

            // if we have a url, try to get the full uri from a relative uri.
			if (linkurl != null) {
                href = linkurl.toString(); // Get full url in case this was a relative url
                logger.debug("Adding URL {} to the list to be processed.", linkurl);
                try {
                    retUris.add(linkurl.toURI());
                } catch (URISyntaxException uriex) {
                    logger.warn("unable to add uri: "+linkurl, uriex);
                }
			}
		}

        return retUris;
    }

    /**
     * @param metadataFields the metadata fields to record
     */
    public void setMetadataFields(Map<String, String> metadataFields) {
        this.metadataFields = metadataFields;
    }
}
